In [2]:
import numpy as np
import pandas as pd
import json
import sys
import os
import matplotlib
matplotlib.use('Agg') 
import matplotlib.pyplot as plt
import seaborn as sns
import pdb

from util import utils as data_utils

%pylab inline
%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'Blues'

# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2

json_file = './cifar_results/noise_45/no_bootstrap_lr_001/checkpoint_100.json'
FDIR = os.path.dirname(json_file)
NUM_CLASSIFY = 10
Populating the interactive namespace from numpy and matplotlib
The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
/usr/local/lib/python2.7/dist-packages/matplotlib/__init__.py:1350: UserWarning:  This call to matplotlib.use() has no effect
because the backend has already been chosen;
matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
or matplotlib.backends is imported for the first time.

  warnings.warn(_use_error_msg)
In [ ]:
 
In [3]:
# Plot gradients norms for the entire learning process
grads_json_filename = os.path.join(FDIR, 'model_grads.json')
grads = [[], [], []]
grads_key = ['max_grad_w1_16', 'max_grad_w1_32', 'max_grad_w1_64']
if os.path.exists(grads_json_filename):
    with open(grads_json_filename, 'r') as fp:
        data = json.load(fp)
        for i, k in enumerate(grads_key):
            if data[0].get(k, None) is None:
                continue
            for batch_grads in data:
                grads[i].append(batch_grads[k])

def plot_grads(grads, title, x_label, y_label, figsize=(10, 8)):
    plt.figure(figsize=figsize)
    # plt.subplot(2, 1, 1)
    plt.plot(grads)
    plt.title(title)
    plt.ylabel(y_label)
    plt.xlabel(x_label)
    
for i, g in enumerate(grads):
    if len(g) > 0:
        plot_grads(g, grads_key[i], 'iterations', grads_key[i])
        # pass
In [4]:
with open(json_file, 'r') as fp:
    data = json.load(fp)
# Loss history might not be of equal length.
train_loss_hist = data['train_loss_history']
val_loss_hist = data['val_loss_history']

# pdb.set_trace()
def plot_loss_hist(loss_hist, title,):
    plt.figure(figsize=(5,4))
    plt.subplot(1, 1, 1)
    plt.plot(loss_hist)
    plt.title(title)  # Train Loss
    plt.ylabel('loss')
    plt.xlabel('time')
    plt.show()
    
plot_loss_hist(train_loss_hist, 'Train Loss')
plot_loss_hist(val_loss_hist, 'Val loss')

if data.get('crit1_loss_history', None) is not None:
    plot_loss_hist(data['crit1_loss_history'], 'Target criterion loss')

if data.get('crit2_loss_history', None) is not None and \
    len(data['crit2_loss_history']) > 0:
    plot_loss_hist(data['crit2_loss_history'], 'Pred criterion loss')

if data.get('pred_loss_history', None) is not None and \
    len(data['pred_loss_history']) > 0:
    plot_loss_hist(data['pred_loss_history'], 'Total Pred loss (beta*t + (1-beta)*p)')    

if data.get('beta_loss_history', None) is not None and \
    len(data['beta_loss_history']) > 0:
    plot_loss_hist(data['beta_loss_history'], 'Beta loss')
In [5]:
if data.get('KL_loss_history', None) is not None:
    # Loss history might not be of equal length.
    KL_loss_hist = data['KL_loss_history']

    plt.figure(figsize=(10,8))
    plt.plot(KL_loss_hist)
    plt.title('KL loss')
    plt.ylabel('loss')
    plt.xlabel('time')
    plt.show()
In [6]:
def get_conf(json_file, num_classes=26, json_key='conf'):
    with open(json_file, 'r') as fp:
        data = json.load(fp)
        conf = data.get(json_key, None)
    if conf is None:
        return
    # c1 = conf.split('\n')[1].split("]")[0].split("[ ")[1].split(" ")
    c1 = conf.split('\n')
    # print(c1)
    conf_mat, row_idx = np.zeros((num_classes, num_classes)), 0
    for i in c1:
        #pdb.set_trace()
        is_conf_row = False
        if ']' in i and '[[' in i:
            val = i.split(']')[0].split('[[')[1].split(' ')
            is_conf_row = True
        elif ']' in i and '[' in i:
            val = i.split(']')[0].split('[')[1].split(' ')
            is_conf_row = True
        if is_conf_row:
            col_idx = 0
            for v in val:
                if not len(v):
                    continue
                try:
                    conf_mat[row_idx, col_idx] = int(v)
                    col_idx = col_idx + 1
                except:
                    continue
            row_idx = row_idx + 1
    
    assert(row_idx == num_classes)
    conf_mat = conf_mat.astype(int)
    fdir = os.path.dirname(json_file)
    json_name = os.path.basename(json_file)[:-5]
    conf_file_name = fdir + '/' + 'conf_' + json_name + '.txt'
    np.savetxt(conf_file_name, conf_mat, fmt='%d', delimiter=', ')
    return conf_mat


def plot_conf(norm_conf):
  # Plot using seaborn
  # (this is style I used for ResNet matrix)
  plt.figure(figsize=(10,6))
  df_cm = pd.DataFrame(norm_conf)
  sns.heatmap(df_cm, annot=True, cmap="Blues")
  plt.show()
In [7]:
def get_sorted_checkpoints(fdir):
    # Checkpoint files are named as 'checkpoint_%d.json'
    checkpoint_map = {}
    for f in os.listdir(fdir):
        if f.endswith('json') and f.startswith('checkpoint'):
            checkpoint_num = int(f.split('checkpoint_')[-1].split('.')[0])
            checkpoint_map[checkpoint_num] = f
    sorted_checkpoints = []
    for k in sorted(checkpoint_map.keys()):
        v = checkpoint_map[k]
        sorted_checkpoints.append(v)
    return sorted_checkpoints
In [8]:
def best_f_scores(fdir, num_classes=5): 
    best_checkpoints = [None, None, None]
    best_3_fscores = [0, 0, 0]
    best_confs = [np.array(()), np.array(()), np.array(())]
    f1_weight_list = [1.0] * num_classes
    f1_weights = np.array(f1_weight_list)
    sorted_checkpoint_files = get_sorted_checkpoints(fdir)
    for f in sorted_checkpoint_files:
        json_file = fdir + '/' + f
        conf = get_conf(json_file, num_classes, json_key='val_conf')
        norm_conf = data_utils.normalize_conf(conf)
        f1 = data_utils.get_f1_score(conf, f1_weights)
        kappa = data_utils.computeKappa(conf)
        wt_f1 = data_utils.computeWeightedF1(conf)
        print('file: {}, f1: {:.3f}, kappa: {:.3f}, weighted-F1: {:.3f}'.format(
                f, f1, kappa, wt_f1))
        plot_conf(norm_conf)
        max_idx = -1
        for i in range(len(best_3_fscores)):
            if best_3_fscores[i] > f1:
                break
            max_idx = i
        for j in range(max_idx):
            best_3_fscores[j] = best_3_fscores[j+1]
            best_confs[j] = best_confs[j+1]
            best_checkpoints[j] = best_checkpoints[j+1]

        best_3_fscores[max_idx] = f1
        best_confs[max_idx] = conf
        best_checkpoints[max_idx] = f

    return best_3_fscores, best_confs, best_checkpoints
In [9]:
def plot_train_conf(fdir, num_classes=5):
    sorted_checkpoint_files = get_sorted_checkpoints(fdir)
    if len(sorted_checkpoint_files) > 0:
        last_checkpoint = sorted_checkpoint_files[-1]
        json_file = fdir + '/' + last_checkpoint
        conf = get_conf(json_file, num_classes=num_classes, json_key='train_conf')
        print(conf)
        norm_conf = data_utils.normalize_conf(conf)
        f1_weight_list = [1.0] * num_classes
        f1_weights = np.array(f1_weight_list)
        f1 = data_utils.get_f1_score(conf, f1_weights)
        kappa = data_utils.computeKappa(conf)
        wt_f1 = data_utils.computeWeightedF1(conf)
        print('file: {}, f1: {:.3f}, kappa: {:.3f}, weighted-F1: {:.3f}'.format(
            f, f1, kappa, wt_f1))
        plot_conf(norm_conf)

plot_train_conf(FDIR, num_classes=NUM_CLASSIFY)
[[3230   13  110   59   24   13 1433   15   66   60]
 [  32 3412    1    8    0    3    0    8   31 1464]
 [ 100    3 2747  215   81 1654   81   54    7   23]
 [  47    7   55 2919   54  387   77   32 1438   28]
 [  20    0   84   89 3126  104  104 1445    6   10]
 [  44    7 1225  378   99 3118   51   92    5   16]
 [  45   14  107  103 1326   63 3251   37    2    1]
 [  11 1451    7   46   73   82    3 3278   10   51]
 [  49   13   28 1261   16  150   32   17 3350   40]
 [1451  107   18   16    7    9    6   11   61 3383]]
file: <built-in method f of mtrand.RandomState object at 0x7f40522f3230>, f1: 0.636, kappa: 0.421, weighted-F1: 0.636
In [10]:
best_f_scores(FDIR, num_classes=NUM_CLASSIFY)
file: checkpoint_1.json, f1: 0.146, kappa: 0.000, weighted-F1: 0.146
file: checkpoint_2.json, f1: 0.286, kappa: 0.061, weighted-F1: 0.286
file: checkpoint_3.json, f1: 0.433, kappa: 0.323, weighted-F1: 0.433
file: checkpoint_4.json, f1: 0.573, kappa: 0.559, weighted-F1: 0.573
file: checkpoint_5.json, f1: 0.603, kappa: 0.581, weighted-F1: 0.603
file: checkpoint_6.json, f1: 0.687, kappa: 0.643, weighted-F1: 0.687
file: checkpoint_7.json, f1: 0.741, kappa: 0.743, weighted-F1: 0.741
file: checkpoint_8.json, f1: 0.722, kappa: 0.644, weighted-F1: 0.722
file: checkpoint_9.json, f1: 0.756, kappa: 0.741, weighted-F1: 0.756
file: checkpoint_10.json, f1: 0.770, kappa: 0.781, weighted-F1: 0.770
file: checkpoint_11.json, f1: 0.765, kappa: 0.767, weighted-F1: 0.765
file: checkpoint_12.json, f1: 0.795, kappa: 0.807, weighted-F1: 0.795
file: checkpoint_13.json, f1: 0.799, kappa: 0.782, weighted-F1: 0.799
file: checkpoint_14.json, f1: 0.812, kappa: 0.813, weighted-F1: 0.812
file: checkpoint_15.json, f1: 0.820, kappa: 0.819, weighted-F1: 0.820
file: checkpoint_16.json, f1: 0.826, kappa: 0.829, weighted-F1: 0.826
file: checkpoint_17.json, f1: 0.820, kappa: 0.819, weighted-F1: 0.820
file: checkpoint_18.json, f1: 0.823, kappa: 0.814, weighted-F1: 0.823
file: checkpoint_19.json, f1: 0.819, kappa: 0.808, weighted-F1: 0.819
file: checkpoint_20.json, f1: 0.848, kappa: 0.849, weighted-F1: 0.848
file: checkpoint_21.json, f1: 0.831, kappa: 0.837, weighted-F1: 0.831
file: checkpoint_22.json, f1: 0.832, kappa: 0.831, weighted-F1: 0.832
file: checkpoint_23.json, f1: 0.851, kappa: 0.841, weighted-F1: 0.851
file: checkpoint_24.json, f1: 0.856, kappa: 0.853, weighted-F1: 0.856
file: checkpoint_25.json, f1: 0.861, kappa: 0.860, weighted-F1: 0.861
file: checkpoint_26.json, f1: 0.859, kappa: 0.858, weighted-F1: 0.859
file: checkpoint_27.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_28.json, f1: 0.862, kappa: 0.862, weighted-F1: 0.862
file: checkpoint_29.json, f1: 0.858, kappa: 0.857, weighted-F1: 0.858
file: checkpoint_30.json, f1: 0.856, kappa: 0.855, weighted-F1: 0.856
file: checkpoint_31.json, f1: 0.862, kappa: 0.860, weighted-F1: 0.862
file: checkpoint_32.json, f1: 0.859, kappa: 0.860, weighted-F1: 0.859
file: checkpoint_33.json, f1: 0.858, kappa: 0.855, weighted-F1: 0.858
file: checkpoint_34.json, f1: 0.859, kappa: 0.859, weighted-F1: 0.859
file: checkpoint_35.json, f1: 0.859, kappa: 0.862, weighted-F1: 0.859
file: checkpoint_36.json, f1: 0.857, kappa: 0.860, weighted-F1: 0.857
file: checkpoint_37.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_38.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_39.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_40.json, f1: 0.859, kappa: 0.859, weighted-F1: 0.859
file: checkpoint_41.json, f1: 0.861, kappa: 0.860, weighted-F1: 0.861
file: checkpoint_42.json, f1: 0.859, kappa: 0.857, weighted-F1: 0.859
file: checkpoint_43.json, f1: 0.860, kappa: 0.861, weighted-F1: 0.860
file: checkpoint_44.json, f1: 0.861, kappa: 0.861, weighted-F1: 0.861
file: checkpoint_45.json, f1: 0.861, kappa: 0.859, weighted-F1: 0.861
file: checkpoint_46.json, f1: 0.858, kappa: 0.855, weighted-F1: 0.858
file: checkpoint_47.json, f1: 0.860, kappa: 0.859, weighted-F1: 0.860
file: checkpoint_48.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_49.json, f1: 0.860, kappa: 0.856, weighted-F1: 0.860
file: checkpoint_50.json, f1: 0.860, kappa: 0.857, weighted-F1: 0.860
file: checkpoint_51.json, f1: 0.862, kappa: 0.861, weighted-F1: 0.862
file: checkpoint_52.json, f1: 0.858, kappa: 0.858, weighted-F1: 0.858
file: checkpoint_53.json, f1: 0.859, kappa: 0.858, weighted-F1: 0.859
file: checkpoint_54.json, f1: 0.857, kappa: 0.857, weighted-F1: 0.857
file: checkpoint_55.json, f1: 0.860, kappa: 0.858, weighted-F1: 0.860
file: checkpoint_56.json, f1: 0.859, kappa: 0.859, weighted-F1: 0.859
file: checkpoint_57.json, f1: 0.859, kappa: 0.857, weighted-F1: 0.859
file: checkpoint_58.json, f1: 0.858, kappa: 0.858, weighted-F1: 0.858
file: checkpoint_59.json, f1: 0.860, kappa: 0.857, weighted-F1: 0.860
file: checkpoint_60.json, f1: 0.859, kappa: 0.858, weighted-F1: 0.859
file: checkpoint_61.json, f1: 0.859, kappa: 0.856, weighted-F1: 0.859
file: checkpoint_62.json, f1: 0.860, kappa: 0.861, weighted-F1: 0.860
file: checkpoint_63.json, f1: 0.859, kappa: 0.859, weighted-F1: 0.859
file: checkpoint_64.json, f1: 0.860, kappa: 0.859, weighted-F1: 0.860
file: checkpoint_65.json, f1: 0.861, kappa: 0.860, weighted-F1: 0.861
file: checkpoint_66.json, f1: 0.858, kappa: 0.857, weighted-F1: 0.858
file: checkpoint_67.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_68.json, f1: 0.862, kappa: 0.861, weighted-F1: 0.862
file: checkpoint_69.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_70.json, f1: 0.859, kappa: 0.858, weighted-F1: 0.859
file: checkpoint_71.json, f1: 0.859, kappa: 0.856, weighted-F1: 0.859
file: checkpoint_72.json, f1: 0.862, kappa: 0.860, weighted-F1: 0.862
file: checkpoint_73.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_74.json, f1: 0.861, kappa: 0.863, weighted-F1: 0.861
file: checkpoint_75.json, f1: 0.860, kappa: 0.859, weighted-F1: 0.860
file: checkpoint_76.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_77.json, f1: 0.860, kappa: 0.859, weighted-F1: 0.860
file: checkpoint_78.json, f1: 0.860, kappa: 0.856, weighted-F1: 0.860
file: checkpoint_79.json, f1: 0.861, kappa: 0.861, weighted-F1: 0.861
file: checkpoint_80.json, f1: 0.858, kappa: 0.857, weighted-F1: 0.858
file: checkpoint_81.json, f1: 0.861, kappa: 0.862, weighted-F1: 0.861
file: checkpoint_82.json, f1: 0.860, kappa: 0.858, weighted-F1: 0.860
file: checkpoint_83.json, f1: 0.859, kappa: 0.860, weighted-F1: 0.859
file: checkpoint_84.json, f1: 0.859, kappa: 0.860, weighted-F1: 0.859
file: checkpoint_85.json, f1: 0.858, kappa: 0.860, weighted-F1: 0.858
file: checkpoint_86.json, f1: 0.860, kappa: 0.856, weighted-F1: 0.860
file: checkpoint_87.json, f1: 0.859, kappa: 0.859, weighted-F1: 0.859
file: checkpoint_88.json, f1: 0.857, kappa: 0.857, weighted-F1: 0.857
file: checkpoint_89.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
file: checkpoint_90.json, f1: 0.858, kappa: 0.857, weighted-F1: 0.858
file: checkpoint_91.json, f1: 0.858, kappa: 0.857, weighted-F1: 0.858
file: checkpoint_92.json, f1: 0.861, kappa: 0.860, weighted-F1: 0.861
file: checkpoint_93.json, f1: 0.857, kappa: 0.858, weighted-F1: 0.857
file: checkpoint_94.json, f1: 0.861, kappa: 0.862, weighted-F1: 0.861
file: checkpoint_95.json, f1: 0.861, kappa: 0.861, weighted-F1: 0.861
file: checkpoint_96.json, f1: 0.862, kappa: 0.861, weighted-F1: 0.862
file: checkpoint_97.json, f1: 0.860, kappa: 0.861, weighted-F1: 0.860
file: checkpoint_98.json, f1: 0.861, kappa: 0.860, weighted-F1: 0.861
file: checkpoint_99.json, f1: 0.857, kappa: 0.856, weighted-F1: 0.857
file: checkpoint_100.json, f1: 0.860, kappa: 0.860, weighted-F1: 0.860
Out[10]:
([0.86073021427822827, 0.86112775135935871, 0.8603650664895991],
 [array([[911,   1,  25,   6,   7,   0,   6,   3,  31,  10],
         [ 13, 932,   1,   5,   0,   1,   1,   2,  19,  26],
         [ 39,   0, 834,  26,  23,  48,  26,   3,   0,   1],
         [ 17,   2,  50, 762,  21,  84,  45,   2,  13,   4],
         [  6,   1,  47,  25, 839,  14,  57,   8,   3,   0],
         [  6,   0,  63, 140,  25, 733,  16,  17,   0,   0],
         [ 10,   2,  24,  21,   4,   4, 933,   2,   0,   0],
         [  6,   0,  13,  38,  47,  30,   1, 862,   0,   3],
         [ 33,   6,   8,  24,   3,   1,   5,   0, 910,  10],
         [ 28,  48,   1,  10,   2,   2,   2,   0,  19, 888]]),
  array([[911,   1,  25,   7,   6,   0,   5,   4,  31,  10],
         [ 12, 943,   1,   4,   0,   1,   1,   2,  16,  20],
         [ 40,   0, 825,  26,  22,  57,  26,   3,   0,   1],
         [ 17,   2,  44, 764,  19,  90,  44,   3,  13,   4],
         [  7,   1,  49,  25, 833,  15,  59,   8,   3,   0],
         [  7,   0,  58, 141,  25, 735,  16,  17,   0,   1],
         [ 11,   2,  22,  21,   3,   4, 935,   2,   0,   0],
         [  6,   0,  12,  38,  40,  31,   1, 869,   0,   3],
         [ 32,   8,   8,  27,   2,   1,   4,   0, 906,  12],
         [ 27,  52,   1,  10,   1,   1,   2,   1,  18, 887]]),
  array([[914,   1,  24,   6,   7,   0,   5,   3,  30,  10],
         [ 13, 934,   1,   4,   0,   1,   1,   2,  19,  25],
         [ 39,   0, 827,  27,  24,  54,  25,   3,   0,   1],
         [ 17,   2,  44, 768,  21,  86,  43,   2,  13,   4],
         [  7,   1,  47,  25, 837,  15,  57,   8,   3,   0],
         [  7,   0,  60, 143,  25, 733,  15,  17,   0,   0],
         [ 12,   2,  21,  21,   4,   5, 933,   2,   0,   0],
         [  6,   0,  12,  40,  47,  31,   1, 860,   0,   3],
         [ 33,   7,   7,  27,   3,   1,   4,   0, 907,  11],
         [ 29,  50,   1,  10,   2,   1,   2,   0,  18, 887]])],
 ['checkpoint_95.json', 'checkpoint_94.json', 'checkpoint_100.json'])